[ { "memory": 3058188544, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3060547840, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3062907136, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3063595264, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3064283392, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3072213248, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3080143104, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3088072960, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3096002816, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3103932672, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3111862528, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3119792384, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3120480512, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3122839808, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3130769664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3131457792, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3133817088, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3134505216, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3142435072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3143123200, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3151053056, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3151741184, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3154100480, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3156459776, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3158819072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3166748928, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3174678784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3182608640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3190538496, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3191226624, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3193585920, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3201515776, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3209445632, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3211804928, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3212493056, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3214852352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3222782208, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3225141504, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3227500800, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 3235430656, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3243360512, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3245719808, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3253649664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3261579520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3269509376, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3277439232, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3285369088, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3293298944, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3293987072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3301916928, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3309846784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3317776640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3318464768, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3319152896, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3321512192, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3329442048, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3337371904, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3339731200, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3340419328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3348349184, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3356279040, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3364208896, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3372138752, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3380068608, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3380756736, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3381444864, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3382132992, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3382821120, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3390750976, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3391439104, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3399368960, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3407298816, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3407986944, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3415916800, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3423846656, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3426205952, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3434135808, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3442065664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3444424960, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3452354816, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3453042944, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3460972800, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3461660928, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3469590784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3477520640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3478208768, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3480568064, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3482927360, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3485286656, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3493216512, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3495575808, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3497935104, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3498623232, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3499311360, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3501670656, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3502358784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3510288640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3510976768, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3511664896, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3514024192, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3521954048, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3522642176, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3525001472, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3527360768, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3529720064, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3532079360, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3540009216, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3547939072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3548627200, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3556557056, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3558916352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3566846208, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3567534336, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3569893632, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3577823488, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3585753344, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3588112640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3596042496, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3598401792, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3599089920, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3601449216, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3609379072, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3611738368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3619668224, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 3627598080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3628286208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3628974336, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3629662464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3630350592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3631038720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3633398016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3641327872, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3642016000, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3649945856, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3652305152, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3654664448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3657023744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3657711872, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3660071168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3662430464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3663118592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3665477888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3667837184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3668525312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3676455168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3677143296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3679502592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3680190720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3688120576, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3696050432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3696738560, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3704668416, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3712598272, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3713286400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3721216256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3729146112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3737075968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3737764096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3740123392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3742482688, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3750412544, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3758342400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3759030528, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3766960384, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3767648512, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3768336640, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3769024768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3771384064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3779313920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3787243776, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3795173632, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3795861760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3803791616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3811721472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3814080768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3822010624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3824369920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3832299776, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3840229632, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3840917760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3843277056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3851206912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3859136768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3867066624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3874996480, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3875684608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3883614464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3884302592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3886661888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3894591744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3896951040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3904880896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3907240192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3907928320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3908616448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3910975744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3918905600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3926835456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3934765312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3937124608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3939483904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3941843200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3949773056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3957702912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3965632768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3966320896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3968680192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3971039488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3971727616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3979657472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3982016768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3982704896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3983393024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3984081152, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3986440448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3987128576, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3995058432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 3997417728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4005347584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4006035712, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4008395008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4009083136, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4011442432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4012130560, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4014489856, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4015177984, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4017537280, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4025467136, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4033396992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4035756288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4038115584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4040474880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4048404736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4056334592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4058693888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4061053184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4063412480, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4071342336, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4079272192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4087202048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4095131904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4103061760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4110991616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4111679744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4119609600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4127539456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4129898752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4137828608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4138516736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4140876032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4148805888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4156735744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4164665600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4172595456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4174954752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4182884608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4190814464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4198744320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4201103616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4203462912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4211392768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4212080896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4220010752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4227940608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4230299904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4238229760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4238917888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4246847744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4254777600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4262707456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4265066752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4272996608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4280926464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4283285760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4283973888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4286333184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4287021312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4287709440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4288397568, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4289085696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4297015552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4297703680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4305633536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4307992832, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4315922688, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4318281984, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4318970112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4326899968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4329259264, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4337189120, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4345118976, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4347478272, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4348166400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4350525696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4358455552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4359143680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4359831808, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4362191104, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4364550400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4365238528, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4367597824, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4368285952, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4368974080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4376903936, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 4384833792, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4387193088, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4395122944, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4397482240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4405412096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4406100224, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4414030080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4414718208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4417077504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4419436800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4420124928, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 2 } }, { "memory": 4428054784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4428742912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4431102208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4439032064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4446961920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4447650048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4455579904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4463509760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4471439616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4473798912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4481728768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4484088064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4484776192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4485464320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4486152448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4488511744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4490871040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4493230336, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4501160192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4501848320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4509778176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4510466304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4512825600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4520755456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4523114752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4531044608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4531732736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4539662592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4547592448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4555522304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4563452160, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4564140288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4572070144, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4580000000, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4587929856, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4595859712, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4596547840, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4598907136, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4606836992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4614766848, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4617126144, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4617814272, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4625744128, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4633673984, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4634362112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4636721408, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4644651264, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4652581120, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4660510976, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4662870272, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4670800128, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4671488256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4672176384, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4674535680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4682465536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4683153664, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4685512960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4687872256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4695802112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4703731968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4704420096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4706779392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4707467520, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4708155648, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4708843776, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4711203072, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4711891200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4719821056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4720509184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4721197312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4721885440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4722573568, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4724932864, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4732862720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4735222016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4737581312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4745511168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4753441024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4755800320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4758159616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4760518912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4762878208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4770808064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4771496192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4773855488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4781785344, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4789715200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4790403328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4792762624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4793450752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4801380608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4809310464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4809998592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4817928448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4818616576, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4819304704, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4827234560, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4835164416, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4843094272, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4845453568, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4853383424, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4855742720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4856430848, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4857118976, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4857807104, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4860166400, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4860854528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4868784384, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4871143680, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4871831808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4874191104, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4874879232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4877238528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4877926656, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4885856512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4893786368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4896145664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4896833792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4897521920, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4898210048, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4906139904, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4908499200, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4910858496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4913217792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4921147648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4929077504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4929765632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4937695488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4945625344, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4953555200, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4954243328, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4962173184, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4970103040, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4970791168, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4973150464, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4973838592, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4981768448, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4989698304, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4992057600, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4992745728, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 5000675584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4 } }, { "memory": 5008605440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5016535296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5024465152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5032395008, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5040324864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5041012992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5048942848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5056872704, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5064802560, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5072732416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5080662272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5088592128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5090951424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5093310720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5101240576, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5103599872, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5105959168, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5106647296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5114577152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5122507008, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5130436864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5138366720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5146296576, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5154226432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5162156288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5170086144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5178016000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5180375296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5188305152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5196235008, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5196923136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5204852992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5212782848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5215142144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5223072000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5231001856, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5238931712, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5246861568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5254791424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5257150720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5257838848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5258526976, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5260886272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5263245568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5271175424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5279105280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5287035136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5289394432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5297324288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5298012416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5305942272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5313872128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5321801984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5324161280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5332091136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5340020992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5340709120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5343068416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5345427712, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5353357568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5361287424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5369217280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5377147136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5379506432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5381865728, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5389795584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5392154880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5394514176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5402444032, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5410373888, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5418303744, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5418991872, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5426921728, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5434851584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5442781440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5443469568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5451399424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5452087552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5452775680, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5460705536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5468635392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5469323520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5477253376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5479612672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5487542528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5495472384, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5503402240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5511332096, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5519261952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5527191808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5535121664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5543051520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5550981376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5558911232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5559599360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5560287488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5562646784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5570576640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5578506496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5580865792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5588795648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5596725504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5604655360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5607014656, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5614944512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5622874368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5623562496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5625921792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5626609920, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5634539776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5635227904, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5643157760, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5645517056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5653446912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5654135040, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5656494336, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5658853632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5661212928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5669142784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5677072640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5677760768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5680120064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5682479360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5683167488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5685526784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5693456640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5701386496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5709316352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5717246208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5719605504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5720293632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5722652928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5723341056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5724029184, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5724717312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5732647168, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5735006464, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5737365760, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5739725056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5740413184, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5741101312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5743460608, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5751390464, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5759320320, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5760008448, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5767938304, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5775868160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5776556288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5778915584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5786845440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5787533568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5789892864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5790580992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5792940288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5800870144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5808800000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5811159296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5811847424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5814206720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5814894848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5815582976, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5817942272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5825872128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5828231424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5828919552, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5829607680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5831966976, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5834326272, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5836685568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5837373696, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5839732992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5842092288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5842780416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5843468544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5844156672, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5846515968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5847204096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5849563392, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5850251520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5852610816, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5860540672, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5862899968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5863588096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5864276224, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5864964352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5867323648, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5868011776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5868699904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5869388032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5877317888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5878006016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5880365312, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5882724608, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5885083904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5887443200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5888131328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5888819456, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5889507584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5891866880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5894226176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5902156032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5902844160, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5905203456, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5905891584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5906579712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5908939008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5909627136, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5910315264, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5912674560, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5913362688, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5915721984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5916410112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5917098240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5925028096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5925716224, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5926404352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5927092480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 5935022336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 5942952192, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5950882048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5953241344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5961171200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5969101056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5977030912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5984960768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5987320064, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5995249920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5995938048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5998297344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6006227200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6006915328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6014845184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6022775040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6025134336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6025822464, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6033752320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6041682176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6049612032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6057541888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6065471744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6073401600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6081331456, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6082019584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6089949440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6097879296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6105809152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6113739008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6121668864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6124028160, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6131958016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6139887872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6147817728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6155747584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6163677440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6164365568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6165053696, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6167412992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6169772288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6177702144, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6185632000, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6193561856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6201491712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6209421568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6217351424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6225281280, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6233211136, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6241140992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6249070848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6251430144, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6253789440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6261719296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6262407424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6270337280, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6278267136, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6286196992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6286885120, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6289244416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6297174272, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6305104128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6307463424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6309822720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6312182016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6320111872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6328041728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6328729856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6336659712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6337347840, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6345277696, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6353207552, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6361137408, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6369067264, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6369755392, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6377685248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6378373376, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6386303232, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6388662528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6391021824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6398951680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6406881536, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6409240832, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6411600128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6419529984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6427459840, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6435389696, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6436077824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6444007680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6451937536, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6459867392, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6467797248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6475727104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6483656960, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6484345088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6492274944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6500204800, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6502564096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6510493952, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6518423808, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6526353664, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6534283520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6542213376, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6550143232, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6558073088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6558761216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6561120512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6561808640, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6569738496, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6572097792, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6572785920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6575145216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6583075072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6591004928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6591693056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6594052352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6601982208, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6609912064, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6617841920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6620201216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6628131072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6636060928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6638420224, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6640779520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6643138816, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6643826944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6646186240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6646874368, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6649233664, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6649921792, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6657851648, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6665781504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6666469632, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6667157760, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6667845888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6675775744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6678135040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6678823168, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6679511296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6687441152, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6695371008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6697730304, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6700089600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6702448896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6703137024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6711066880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6711755008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6714114304, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6714802432, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6722732288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6723420416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6725779712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6728139008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6728827136, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6731186432, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6739116288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6747046144, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6749405440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6751764736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6754124032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6754812160, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6762742016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6765101312, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6765789440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6768148736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6768836864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6771196160, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6771884288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6772572416, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6774931712, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6777291008, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6777979136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6780338432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6781026560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6781714688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6782402816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6783090944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6785450240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6787809536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6788497664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6790856960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6798786816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6799474944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6801834240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6804193536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6804881664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6807240960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6807929088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6815858944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6818218240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6820577536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6822936832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6825296128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6833225984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6833914112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6834602240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6835290368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6837649664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6840008960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6842368256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6843056384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6843744512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6846103808, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6848463104, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6849151232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6849839360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6850527488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6858457344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6859145472, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6859833600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6860521728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6861209856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6861897984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6862586112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6864945408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6867304704, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6869664000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6872023296, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6872711424, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6873399552, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6881329408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6882017536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6882705664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6883393792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 6891323648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 6 } }, { "memory": 6899253504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6907183360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6915113216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6923043072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6930972928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6931661056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6939590912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6947520768, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6955450624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6957809920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6965739776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6973669632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6976028928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6983958784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6991888640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6999818496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7007748352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7015678208, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7016366336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7024296192, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7024984320, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7027343616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7029702912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7030391040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7038320896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7039009024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7046938880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7054868736, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7055556864, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7063486720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7065846016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7066534144, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7074464000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7076823296, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7079182592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7087112448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7089471744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7097401600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7105331456, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7113261312, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7121191168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7129121024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7137050880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7144980736, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7152910592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7153598720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7161528576, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7169458432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7177388288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7185318144, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7193248000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7201177856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7209107712, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7217037568, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7224967424, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7232897280, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7233585408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7241515264, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7249445120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7257374976, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7265304832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7273234688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7281164544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7289094400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7297024256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7304954112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7312883968, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7313572096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7321501952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7329431808, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7337361664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7345291520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7353221376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7361151232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7361839360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7364198656, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7372128512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7380058368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7382417664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7383105792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7383793920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7386153216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7386841344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7394771200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7402701056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7405060352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7405748480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7408107776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7416037632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7416725760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7424655616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7427014912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7434944768, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7442874624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7450804480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7458734336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7461093632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7461781760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7464141056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7464829184, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7467188480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7469547776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7477477632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7478165760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7486095616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7488454912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7496384768, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7497072896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7499432192, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7507362048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7508050176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7510409472, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7518339328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7520698624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7521386752, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7529316608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7531675904, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7539605760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7547535616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7549894912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7550583040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7558512896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7566442752, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7567130880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7569490176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7577420032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7585349888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7586038016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7586726144, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7594656000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7595344128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7603273984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7605633280, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7613563136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7614251264, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7614939392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7622869248, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7623557376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7624245504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7626604800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7627292928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7629652224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7630340352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7632699648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7633387776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7641317632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7642005760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7649935616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7650623744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7651311872, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7653671168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7656030464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7656718592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7657406720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7659766016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7667695872, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7675625728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7676313856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7677001984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7677690112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7680049408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7682408704, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7690338560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7692697856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7700627712, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7702987008, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7703675136, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7704363264, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7712293120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7712981248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7715340544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7723270400, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7723958528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7726317824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7734247680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7734935808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7735623936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7737983232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7740342528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7742701824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7743389952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7744078080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7744766208, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7752696064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7760625920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7761314048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7762002176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7764361472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7772291328, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7774650624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7777009920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7777698048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7780057344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7782416640, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7784775936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7785464064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7787823360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7790182656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7792541952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7794901248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7797260544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7799619840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7800307968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7802667264, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7805026560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7805714688, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7808073984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7816003840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7816691968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7817380096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7819739392, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7820427520, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7821115648, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7823474944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7825834240, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7828193536, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7828881664, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7829569792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7831929088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7834288384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7836647680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7839006976, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7839695104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 7847624960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7 } }, { "memory": 7855554816, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7 } }, { "memory": 7863484672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7865843968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7873773824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7876133120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7878492416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7886422272, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7894352128, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7902281984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7910211840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7918141696, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7920500992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7928430848, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7930790144, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7933149440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7941079296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7949009152, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7956939008, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7957627136, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7965556992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7973486848, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7981416704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7989346560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7997276416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7999635712, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8007565568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8015495424, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8023425280, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8024113408, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8024801536, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8032731392, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8040661248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8041349376, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8049279232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8049967360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8050655488, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8058585344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8066515200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8074445056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8082374912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8090304768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8092664064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8100593920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8102953216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8110883072, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8111571200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8113930496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8121860352, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8122548480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8130478336, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8138408192, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8146338048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8154267904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8162197760, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8170127616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8172486912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8180416768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8188346624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8196276480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8204206336, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8212136192, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8212824320, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8220754176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8221442304, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8222130432, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8230060288, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8237990144, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8245920000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8246608128, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8254537984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8262467840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8263155968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8271085824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8271773952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8279703808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8280391936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8288321792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8296251648, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8304181504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8312111360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8320041216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8327971072, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8335900928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8338260224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8346190080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8354119936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8362049792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8362737920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8365097216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8365785344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8373715200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8381645056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8389574912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8397504768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8405434624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8413364480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8415723776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8418083072, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8420442368, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8422801664, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8423489792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8431419648, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8432107776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8440037632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8447967488, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8455897344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8463827200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8464515328, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8466874624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8469233920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8477163776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8479523072, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8487452928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8488141056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8488829184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8496759040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8497447168, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8499806464, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8500494592, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8502853888, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8503542016, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8505901312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8508260608, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8510619904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8512979200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8520909056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8523268352, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8523956480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8531886336, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8532574464, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8540504320, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8542863616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8545222912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8553152768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8555512064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8563441920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8564130048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8564818176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8565506304, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8566194432, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8568553728, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8576483584, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8578842880, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8581202176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8581890304, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8584249600, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8586608896, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8587297024, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8587985152, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8590344448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8591032576, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8591720704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8599650560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8602009856, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8609939712, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8617869568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8618557696, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8620916992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8621605120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8623964416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8624652544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8625340672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8627699968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8628388096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8636317952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8637006080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8637694208, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8640053504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8642412800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8650342656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8651030784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8658960640, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8661319936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8662008064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8669937920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8672297216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8672985344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8673673472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8676032768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8678392064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8686321920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8694251776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8702181632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8704540928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8706900224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8707588352, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8708276480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8708964608, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8711323904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8712012032, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8712700160, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8720630016, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8722989312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8725348608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8733278464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8733966592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8734654720, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8737014016, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8737702144, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8738390272, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8740749568, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8743108864, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8745468160, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8747827456, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8748515584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8750874880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8758804736, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8761164032, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8763523328, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8765882624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8773812480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8776171776, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8776859904, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8779219200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8781578496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8782266624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8784625920, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8785314048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8786002176, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8786690304, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8787378432, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8788066560, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8795996416, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 8803926272, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 8 } }, { "memory": 8811856128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8819785984, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8822145280, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8830075136, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8830763264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8838693120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8846622976, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8847311104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8847999232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8855929088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8856617216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8864547072, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8866906368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8874836224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8882766080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8883454208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8884142336, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8892072192, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8900002048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8907931904, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8915861760, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8923791616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8931721472, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8939651328, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8947581184, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8948269312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8956199168, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8958558464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8966488320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8968847616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8969535744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8977465600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8985395456, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8993325312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9001255168, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9009185024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9017114880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9019474176, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9027404032, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9035333888, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9043263744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9051193600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9053552896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9061482752, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9062170880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9070100736, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9078030592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9085960448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9088319744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9090679040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9091367168, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9099297024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9101656320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9102344448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9104703744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9112633600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9114992896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9122922752, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9130852608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9133211904, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9141141760, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9149071616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9151430912, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9159360768, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9160048896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9160737024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9168666880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9176596736, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9184526592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9186885888, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9194815744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9195503872, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9203433728, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9211363584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9212051712, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9212739840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9213427968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9214116096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9216475392, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9224405248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9225093376, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9233023232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9233711360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9241641216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9242329344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9250259200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9250947328, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9251635456, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9259565312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9267495168, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9275425024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9276113152, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9284043008, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9291972864, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9292660992, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9293349120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9301278976, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9301967104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9309896960, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9312256256, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9320186112, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9328115968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9330475264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9338405120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9339093248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9347023104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9347711232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9348399360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9350758656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9358688512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9366618368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9368977664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9376907520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9377595648, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9385525504, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9386213632, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9388572928, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9389261056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9391620352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9399550208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9400238336, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9402597632, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9410527488, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9418457344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9426387200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9428746496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9429434624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9437364480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9445294336, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9447653632, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9448341760, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9450701056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9451389184, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9453748480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9456107776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9458467072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9459155200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9467085056, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9475014912, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9475703040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9478062336, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9480421632, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9482780928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9485140224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9485828352, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9486516480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9487204608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9487892736, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9488580864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9490940160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9493299456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9493987584, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9494675712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9502605568, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9504964864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9507324160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9509683456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9510371584, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9512730880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9515090176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9517449472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9519808768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9527738624, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9535668480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9538027776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9545957632, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9546645760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9549005056, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9551364352, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9553723648, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9556082944, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9556771072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9564700928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9565389056, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9567748352, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9568436480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9569124608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9577054464, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9577742592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9585672448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9588031744, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9588719872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9596649728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9597337856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9605267712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9613197568, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9621127424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9623486720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9624174848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9632104704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9632792832, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9633480960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9635840256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9638199552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9646129408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9646817536, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9654747392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9657106688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9665036544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9672966400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9675325696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9677684992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9685614848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9687974144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9690333440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9698263296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9706193152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9708552448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9710911744, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9713271040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9713959168, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9716318464, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9724248320, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9732178176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9740108032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9740796160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9741484288, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9742172416, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9744531712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9746891008, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9749250304, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9749938432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9752297728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 9760227584, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9 } }, { "memory": 9768157440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9 } }, { "memory": 9776087296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9778446592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9780805888, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9783165184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9791095040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9799024896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9806954752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9814884608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9822814464, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9825173760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9833103616, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9841033472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9843392768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9851322624, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9859252480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9859940608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9862299904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9870229760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9878159616, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9886089472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9894019328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9901949184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9909879040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9917808896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9925738752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9926426880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9934356736, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9942286592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9950216448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9958146304, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9966076160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9966764288, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9969123584, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9977053440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9984983296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9992913152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10000843008, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10008772864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10016702720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10024632576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10032562432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10040492288, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10048422144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10049110272, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10057040128, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10064969984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10072899840, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10080829696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10088759552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10096689408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10104619264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10105307392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10113237248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10115596544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10123526400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10131456256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10139386112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10140074240, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10148004096, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10155933952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10156622080, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10158981376, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10166911232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10174841088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10182770944, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10190700800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10198630656, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10206560512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10214490368, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10222420224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10230350080, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10232709376, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10235068672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10235756800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10243686656, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10244374784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10246734080, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10254663936, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10262593792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10270523648, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10272882944, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10280812800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10281500928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10282189056, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10284548352, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10285236480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10285924608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10288283904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10288972032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10296901888, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10304831744, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10312761600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10320691456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10323050752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10323738880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10326098176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10326786304, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10327474432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10335404288, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10336092416, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10338451712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10346381568, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10348740864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10349428992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10350117120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10350805248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10358735104, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10359423232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10361782528, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10364141824, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10366501120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10368860416, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10369548544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10371907840, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10374267136, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10374955264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10377314560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10378002688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10378690816, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10386620672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10387308800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10389668096, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10397597952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10399957248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10400645376, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10401333504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10409263360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10411622656, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10419552512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10421911808, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10424271104, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10432200960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10434560256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10435248384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10443178240, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10445537536, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10446225664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10446913792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10449273088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10451632384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10453991680, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10456350976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10458710272, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10466640128, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10467328256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10468016384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10468704512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10471063808, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10478993664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10481352960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10483712256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10484400384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10485088512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10485776640, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10486464768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10488824064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10489512192, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10490200320, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10490888448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10491576576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10499506432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10500194560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10500882688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10508812544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10509500672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10511859968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10512548096, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10514907392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10517266688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10519625984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10520314112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10522673408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10530603264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10531291392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10533650688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10534338816, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10536698112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10544627968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10552557824, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10554917120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10555605248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10557964544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10565894400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10573824256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10581754112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10589683968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10590372096, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10591060224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10593419520, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10595778816, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10598138112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10606067968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10613997824, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10621927680, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10629857536, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10630545664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10638475520, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10640834816, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10641522944, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10643882240, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10646241536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10648600832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10656530688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10658889984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10659578112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10661937408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10664296704, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10666656000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10667344128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10668032256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10668720384, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10671079680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10679009536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10679697664, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10682056960, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10682745088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10690674944, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10698604800, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10706534656, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10707222784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10707910912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10708599040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 10716528896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 10 } }, { "memory": 10724458752, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10732388608, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10740318464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10748248320, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10756178176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10756866304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10764796160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10772726016, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10773414144, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10781344000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10789273856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10791633152, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10792321280, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10800251136, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10800939264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10808869120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10811228416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10813587712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10821517568, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10823876864, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10831806720, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10832494848, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10834854144, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10835542272, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10843472128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10845831424, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10853761280, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10861691136, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10862379264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10870309120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10878238976, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10878927104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10881286400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10889216256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10897146112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10905075968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10913005824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10920935680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10928865536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10929553664, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10931912960, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10939842816, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10947772672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10955702528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10963632384, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10964320512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10972250368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10972938496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10973626624, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10981556480, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10989486336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10991845632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10992533760, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11000463616, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11001151744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11009081600, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11009769728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11010457856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11012817152, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11013505280, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11014193408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11022123264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11022811392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11030741248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11033100544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11035459840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11043389696, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11051319552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11059249408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11067179264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11075109120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11077468416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11079827712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11080515840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11081203968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11089133824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11097063680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11099422976, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11107352832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11109712128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11117641984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11118330112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11126259968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11134189824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11142119680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11150049536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11157979392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11160338688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11168268544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11176198400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11184128256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11192058112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11194417408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11195105536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11203035392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11210965248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11218895104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11219583232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11221942528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11222630656, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11223318784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11225678080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11228037376, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11235967232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11236655360, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11244585216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11245273344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11247632640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11249991936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11257921792, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11265851648, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11266539776, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11274469632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11282399488, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11283087616, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11291017472, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11298947328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11301306624, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11303665920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11304354048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11312283904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11320213760, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11328143616, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11336073472, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11344003328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11351933184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11352621312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11360551168, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11361239296, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11363598592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11365957888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11368317184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11369005312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11371364608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11379294464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11379982592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11382341888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11384701184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11385389312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11393319168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11395678464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11396366592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11397054720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11397742848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11398430976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11399119104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11407048960, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11414978816, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11417338112, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11425267968, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11433197824, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11435557120, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11437916416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11440275712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11440963840, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11443323136, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11444011264, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11444699392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11452629248, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11454988544, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11455676672, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11456364800, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11458724096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11461083392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11463442688, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11465801984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11466490112, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11468849408, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11469537536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11471896832, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11474256128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11476615424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11477303552, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11477991680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11480350976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11481039104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11481727232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11489657088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11492016384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11492704512, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11500634368, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11502993664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11503681792, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11504369920, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11512299776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11520229632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11520917760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11523277056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11523965184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11526324480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11534254336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11536613632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11538972928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11541332224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11543691520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11544379648, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11552309504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11552997632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11553685760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11554373888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11556733184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11559092480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11561451776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11562139904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11564499200, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11572429056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11580358912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11581047040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11588976896, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11589665024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11597594880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11599954176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11607884032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11615813888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11623743744, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11626103040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11634032896, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11641962752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11649892608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11657822464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11660181760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11662541056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11664900352, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 11672830208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10 } }, { "memory": 11680760064, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11688689920, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11691049216, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11698979072, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11701338368, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11703697664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11711627520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11712315648, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11713003776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11720933632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11728863488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11736793344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11739152640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11747082496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11755012352, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11755700480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11763630336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11771560192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11779490048, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11787419904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11789779200, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11797709056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11805638912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11813568768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11821498624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11829428480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11837358336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11838046464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11845976320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11853906176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11856265472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11864195328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11872125184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11880055040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11887984896, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11895914752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11898274048, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11898962176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11901321472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11902009600, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11909939456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11912298752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11912986880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11920916736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11928846592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11936776448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11944706304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11952636160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11953324288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11961254144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11969184000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11969872128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11970560256, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11971248384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11971936512, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11972624640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11974983936, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11982913792, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11983601920, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11991531776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11999461632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12000149760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12008079616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12016009472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12023939328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12024627456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12032557312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12040487168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12041175296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12049105152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12057035008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12057723136, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12060082432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12068012288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12068700416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12071059712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12078989568, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12086919424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12087607552, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12088295680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12090654976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12093014272, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12100944128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12101632256, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12109562112, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12110250240, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12118180096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12120539392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12128469248, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12130828544, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12133187840, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12133875968, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12134564096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12142493952, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12150423808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12151111936, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12159041792, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12161401088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12169330944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12177260800, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12177948928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12185878784, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12193808640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12196167936, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12204097792, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12212027648, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12219957504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12220645632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12228575488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12236505344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12237193472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12245123328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12245811456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12248170752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12256100608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12256788736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12259148032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12259836160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12262195456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12270125312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12272484608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12273172736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12275532032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12277891328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12285821184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12286509312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12294439168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12302369024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12304728320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12307087616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12315017472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12315705600, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12323635456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12331565312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12333924608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12341854464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12349784320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12352143616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12352831744, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12355191040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12355879168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12356567296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12357255424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12359614720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12367544576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12369903872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12372263168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12372951296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12375310592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12377669888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12378358016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12379046144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12379734272, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12382093568, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12390023424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12397953280, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12400312576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12401000704, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12408930560, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12411289856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12413649152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12416008448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12423938304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12424626432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12426985728, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12429345024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12431704320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12439634176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12447564032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12455493888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12457853184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12458541312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12460900608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12461588736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12463948032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12464636160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12465324288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12467683584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12468371712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12469059840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12476989696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12484919552, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12485607680, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12493537536, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12495896832, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12498256128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12500615424, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12501303552, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12503662848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12506022144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12508381440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12516311296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12524241152, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12532171008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12534530304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12536889600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12537577728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12539937024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12542296320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12550226176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12550914304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12558844160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12559532288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12560220416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12560908544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12568838400, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12569526528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12571885824, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12574245120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12576604416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12578963712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12586893568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12594823424, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12602753280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12605112576, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12607471872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12608160000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12616089856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12616777984, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12617466112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12618154240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12618842368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12621201664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 12629131520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 12 } }, { "memory": 12637061376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12644991232, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12652921088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12660850944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12668780800, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12669468928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12677398784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12678086912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12680446208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12688376064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12696305920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12696994048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12697682176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12700041472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12702400768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12710330624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12711018752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12718948608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12726878464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12734808320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12742738176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12750668032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12751356160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12759286016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12767215872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12775145728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12783075584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12791005440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12798935296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12799623424, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12807553280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12815483136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12817842432, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12825772288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12833702144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12841632000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12849561856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12850249984, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12858179840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12858867968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12866797824, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12869157120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12871516416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12872204544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12880134400, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12880822528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12888752384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12889440512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12891799808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12894159104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12902088960, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12902777088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12903465216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12904153344, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12906512640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12914442496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12922372352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12930302208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12938232064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12938920192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12946850048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12947538176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12955468032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12963397888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12971327744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12979257600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12981616896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12983976192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12986335488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12988694784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12996624640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13004554496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13006913792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13014843648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13017202944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13017891072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13020250368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13028180224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13030539520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13032898816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13033586944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13035946240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13043876096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13044564224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13046923520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13047611648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13055541504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13063471360, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13065830656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13066518784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13067206912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13067895040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13075824896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13076513024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13077201152, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13079560448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13087490304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13088178432, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13090537728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13091225856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13093585152, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13094273280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13096632576, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13098991872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13101351168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13103710464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13111640320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13113999616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13121929472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13124288768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13124976896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13127336192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13135266048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13135954176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13143884032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13146243328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13154173184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13162103040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13170032896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13170721024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13171409152, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13172097280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13180027136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13187956992, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13195886848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13203816704, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13211746560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13214105856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13222035712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13222723840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13223411968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13231341824, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13232029952, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13239959808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13242319104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13243007232, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13250937088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13258866944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13261226240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13261914368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13269844224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13277774080, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13285703936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13286392064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13294321920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13302251776, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13302939904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13310869760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13313229056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13321158912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13321847040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13322535168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13330465024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13332824320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13333512448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13341442304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13343801600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13346160896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13354090752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13362020608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13362708736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13365068032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13372997888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13380927744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13388857600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13389545728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13390233856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13398163712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13398851840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13399539968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13401899264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13402587392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13403275520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13403963648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13411893504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13414252800, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13414940928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13417300224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13425230080, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13433159936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13433848064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13434536192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13435224320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13437583616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13439942912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13442302208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13442990336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13445349632, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13447708928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13448397056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13450756352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13453115648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13461045504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13463404800, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13464092928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13464781056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13465469184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13466157312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13468516608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13470875904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13478805760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13479493888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13481853184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13484212480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13486571776, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13488931072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13491290368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13493649664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13501579520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13503938816, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13506298112, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13514227968, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13522157824, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13530087680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13532446976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13540376832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13548306688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13556236544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13558595840, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13566525696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13568884992, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13576814848, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13577502976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 13585432832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 13 } }, { "memory": 13593362688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13601292544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13601980672, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13609910528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13612269824, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13612957952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13620887808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13623247104, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13631176960, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13631865088, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13639794944, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13642154240, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13650084096, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13658013952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13658702080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13666631936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13674561792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13676921088, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13684850944, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13685539072, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13686227200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13694157056, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13694845184, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13697204480, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13705134336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13707493632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13708181760, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13716111616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13716799744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13724729600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13732659456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13740589312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13742948608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13750878464, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13758808320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13759496448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13761855744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13769785600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13777715456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13785645312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13793575168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13794263296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13802193152, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13810123008, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13818052864, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13825982720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13826670848, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13827358976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13835288832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13837648128, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13838336256, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13846266112, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13846954240, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13854884096, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13862813952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13863502080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13871431936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13879361792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13887291648, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13887979776, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13888667904, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13896597760, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13904527616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13905215744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13905903872, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13908263168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13910622464, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13911310592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13919240448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13919928576, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13920616704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13921304832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13921992960, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13929922816, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13937852672, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13945782528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13953712384, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13961642240, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13969572096, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13971931392, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13974290688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13982220544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13990150400, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13998080256, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13998768384, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 13999456512, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14001815808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14009745664, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14010433792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14012793088, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14020722944, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14021411072, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14023770368, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14024458496, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14032388352, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14040318208, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14041006336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14048936192, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14056866048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14059225344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14067155200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14075085056, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14083014912, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14090944768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14091632896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14099562752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14101922048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14109851904, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14112211200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14120141056, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14120829184, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14121517312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14129447168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14137377024, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14138065152, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14140424448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14148354304, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14150713600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14151401728, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14159331584, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14167261440, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14175191296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14177550592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14185480448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14187839744, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14190199040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14192558336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14193246464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14201176320, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14203535616, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14211465472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14212153600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14212841728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14215201024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14217560320, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14218248448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14220607744, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14221295872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14223655168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14224343296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14232273152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14234632448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14236991744, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14244921600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14245609728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14247969024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14255898880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14256587008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14257275136, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14259634432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14267564288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14269923584, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14270611712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14278541568, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14279229696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14279917824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14282277120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14290206976, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14290895104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14298824960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14301184256, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14301872384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14304231680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14304919808, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14307279104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14307967232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14315897088, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14323826944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14326186240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14328545536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14336475392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14337163520, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14339522816, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14340210944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14342570240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14350500096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14352859392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14360789248, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14363148544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14365507840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14367867136, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14368555264, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14370914560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14378844416, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14379532544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14380220672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14380908800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14383268096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14391197952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14391886080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14394245376, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14402175232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14404534528, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14412464384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14414823680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14422753536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14425112832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14425800960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14428160256, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14436090112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14438449408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14440808704, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14443168000, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14445527296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14453457152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14461387008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14463746304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14466105600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14474035456, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14476394752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14477082880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14485012736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14492942592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14495301888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14503231744, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14503919872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14504608000, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14512537856, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14520467712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14528397568, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14530756864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14533116160, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14533804288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 14541734144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 14 } }, { "memory": 14549664000, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14557593856, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14558281984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14566211840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14574141696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14576500992, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14584430848, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14585118976, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14593048832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14600978688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14608908544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14616838400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14624768256, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14632698112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14640627968, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14648557824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14650917120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14658846976, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14661206272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14663565568, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14664253696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14672183552, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14672871680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14680801536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14683160832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14691090688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14693449984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14694138112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14702067968, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14702756096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14703444224, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14711374080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14712062208, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14719992064, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14720680192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14728610048, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14729298176, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14737228032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14739587328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14747517184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14755447040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14763376896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14771306752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14779236608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14779924736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14782284032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14784643328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14792573184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14794932480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14795620608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14797979904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14798668032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14806597888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14808957184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14816887040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14817575168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14818263296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14820622592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14828552448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14829240576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14837170432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14845100288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14847459584, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14855389440, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14857748736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14865678592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14873608448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14881538304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14889468160, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14890156288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14898086144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14898774272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14906704128, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14907392256, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14915322112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14916010240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14923940096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14924628224, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14925316352, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14933246208, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14941176064, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14941864192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14949794048, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14950482176, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14952841472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14960771328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14961459456, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14963818752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14971748608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14974107904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14976467200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14977155328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14985085184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 14993015040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15000944896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15003304192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15005663488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15013593344, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15015952640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15016640768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15017328896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15018017024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15018705152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15021064448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15023423744, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15031353600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15039283456, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15041642752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15042330880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15044690176, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15045378304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15046066432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15046754560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15049113856, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15057043712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15059403008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15067332864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15075262720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15083192576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15083880704, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15084568832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15086928128, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15089287424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15091646720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15092334848, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15093022976, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15093711104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15096070400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15096758528, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15099117824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15107047680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15107735808, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15115665664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15118024960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15120384256, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15128314112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15130673408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15131361536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15132049664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15132737792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15140667648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15143026944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15150956800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15158886656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15161245952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15163605248, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15165964544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15168323840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15169011968, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15176941824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15179301120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15181660416, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15184019712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15186379008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15194308864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15196668160, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15204598016, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15212527872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15220457728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15221145856, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15229075712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15231435008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15232123136, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15234482432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15242412288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15243100416, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15243788544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15244476672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15245164800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15245852928, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15246541056, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15248900352, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15251259648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15253618944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15261548800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15269478656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15271837952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15279767808, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15280455936, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15288385792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15289073920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15297003776, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15299363072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15307292928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15309652224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15310340352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15312699648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15320629504, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15321317632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15322005760, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15322693888, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15323382016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15331311872, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15339241728, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15347171584, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15347859712, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15348547840, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15356477696, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15364407552, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15365095680, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15373025536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15375384832, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15383314688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15391244544, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15399174400, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15401533696, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15403892992, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15406252288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15408611584, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15416541440, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15418900736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15426830592, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15429189888, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15431549184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15439479040, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15447408896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15455338752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15463268608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15465627904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15473557760, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15481487616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15489417472, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15490105600, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 15498035456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15505965312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15513895168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15514583296, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15516942592, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15524872448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15532802304, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15533490432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 15541420288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15549350144, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15557280000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15565209856, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15573139712, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15581069568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15583428864, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15591358720, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15599288576, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15607218432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15607906560, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15615836416, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15623766272, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15631696128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15639625984, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15647555840, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15655485696, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15656173824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15664103680, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15672033536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15679963392, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15687893248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15695823104, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15703752960, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15711682816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15714042112, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15721971968, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15729901824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15737831680, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15745761536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15748120832, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15748808960, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15756738816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15759098112, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15767027968, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15767716096, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15775645952, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15776334080, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15784263936, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15792193792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15800123648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15802482944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15810412800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15811100928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15819030784, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15821390080, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15822078208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15830008064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15830696192, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15838626048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15846555904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15848915200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15856845056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15864774912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15867134208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15875064064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15877423360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15885353216, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15886041344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15886729472, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15889088768, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15889776896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15890465024, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15892824320, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15895183616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15903113472, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15911043328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15918973184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15921332480, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15922020608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15929950464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15930638592, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15938568448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15946498304, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15947186432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15955116288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15963046144, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15970976000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15978905856, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15979593984, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15980282112, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15980970240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15988900096, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15991259392, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15993618688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15994306816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15994994944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 15997354240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16005284096, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16005972224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16008331520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16010690816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16013050112, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16013738240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16014426368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16022356224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16024715520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16025403648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16027762944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16030122240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16038052096, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16040411392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16041099520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16049029376, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16049717504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16050405632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16051093760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16059023616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16066953472, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16067641600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16068329728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16069017856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16071377152, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16072065280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16074424576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16082354432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16090284288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16098214144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16100573440, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16102932736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16110862592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16113221888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16113910016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16121839872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16124199168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16124887296, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16127246592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16129605888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16130294016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16138223872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16140583168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16142942464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16143630592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16145989888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16146678016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16154607872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16155296000, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16155984128, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16158343424, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16166273280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16168632576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16170991872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16173351168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16175710464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16178069760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16185999616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16188358912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16189047040, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16189735168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16192094464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16192782592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16195141888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16203071744, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16211001600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16211689728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16212377856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16213065984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16215425280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16217784576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16225714432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16228073728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16230433024, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16231121152, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16233480448, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16241410304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16242098432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16250028288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16250716416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16251404544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16253763840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16261693696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16269623552, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16277553408, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16279912704, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16280600832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16288530688, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16296460544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16297148672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16305078528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16307437824, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16309797120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16310485248, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16312844544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16320774400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16323133696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16325492992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16326181120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16334110976, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16342040832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16349970688, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16350658816, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16353018112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16355377408, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16357736704, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16365666560, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16366354688, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16374284544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16374972672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16382902528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16385261824, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16393191680, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16401121536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16403480832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16411410688, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16413769984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16414458112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16415146240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16415834368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16416522496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16424452352, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16426811648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16427499776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16429859072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16437788928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16438477056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16446406912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 16454336768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16462266624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16462954752, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16465314048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16473243904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16481173760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16489103616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16491462912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16493822208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16501752064, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 16509681920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16517611776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16525541632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16533471488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16535830784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16536518912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16544448768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16552378624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16560308480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16568238336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16568926464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16569614592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16570302720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16578232576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16586162432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16594092288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16602022144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16602710272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16610640128, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16618569984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16626499840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16627187968, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16635117824, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16637477120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16638165248, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16640524544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16648454400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16656384256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16664314112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16672243968, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16674603264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16682533120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16690462976, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16692822272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16695181568, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16703111424, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16711041280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16713400576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16714088704, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16722018560, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16722706688, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16725065984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16725754112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16726442240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16727130368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16735060224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16735748352, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16743678208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16751608064, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16759537920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16767467776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16775397632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16777756928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16780116224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16782475520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16790405376, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16791093504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16791781632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16799711488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16800399616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16801087744, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16801775872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16809705728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16817635584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16819994880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16827924736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16835854592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16836542720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16838902016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16839590144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16840278272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16840966400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16841654528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16842342656, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16850272512, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16858202368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16858890496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16859578624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16867508480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16868196608, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16876126464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16884056320, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16886415616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16894345472, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16895033600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16897392896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16905322752, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16907682048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16915611904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16916300032, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16918659328, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16921018624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16928948480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16931307776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16931995904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16939925760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16942285056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16942973184, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16943661312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16946020608, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16953950464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16956309760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16956997888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16959357184, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16967287040, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16967975168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16970334464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16971022592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16978952448, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16979640576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16981999872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16989929728, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16992289024, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 16992977152, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17000907008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17008836864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17009524992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17010213120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17012572416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17014931712, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17022861568, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17030791424, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17033150720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17033838848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17034526976, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17042456832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17044816128, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17052745984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17055105280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17057464576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17059823872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17062183168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17064542464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17065230592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17067589888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17068278016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17070637312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17071325440, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17073684736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17076044032, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17076732160, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17079091456, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17081450752, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17083810048, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17086169344, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17086857472, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17094787328, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17097146624, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17105076480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17113006336, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17115365632, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17116053760, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17116741888, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17119101184, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17119789312, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17122148608, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17124507904, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17132437760, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17134797056, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17142726912, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17143415040, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17144103168, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17146462464, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17147150592, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17147838720, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17150198016, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17158127872, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17166057728, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17173987584, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17174675712, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17182605568, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17190535424, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17191223552, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17193582848, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17194270976, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17202200832, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17210130688, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17218060544, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17225990400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17233920256, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17234608384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17236967680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17239326976, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17240015104, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17242374400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17243062528, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17250992384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17258922240, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17261281536, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17269211392, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17277141248, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17279500544, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17287430400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17288118528, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17296048384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17303978240, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17311908096, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17314267392, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17322197248, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17330127104, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17330815232, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17331503360, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17339433216, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17347363072, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17348051200, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17350410496, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17358340352, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17366270208, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17374200064, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17382129920, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17390059776, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17392419072, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17400348928, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } }, { "memory": 17402708224, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } } ]